#encoding=utf-8

import requests
import json
from urllib.parse import urlencode
from requests.exceptions import RequestException
import re
from bs4 import BeautifulSoup
#import pymongo
from hashlib import md5
import os
from multiprocessing import Pool


MONGO_URL="localhost"
MONGO_DB="toutiao"
MONGO_TABLE="toutiao"

GROUP_START = 1 # 定义起始循环点
GROUP_END = 20 # 定义终止循环点
KEYWORD="街拍"

headers = {
    'authority': 'www.toutiao.com',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
    'Accept': 'text/javascript, text/html, application/xml, text/xml, */*',
    'Accept-Lannguage': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
}

def get_page_index(offset, keyword):
    """传入要数据,获得详情页面"""
    data = {
        "autoload": "true",
        "count": 20,
        "cur_tab": 1,
        "format": "json",
        "from": "search_tab",
        "pd": "synthesis",
        "en_qc": 1,
        "app_name": "web_search",
        "aid": 24,
        "keyword": keyword,
        "offset": offset,
    }
    #为什么要urlencode: https://www.cnblogs.com/jerrysion/p/5522673.html
    url = " https://www.toutiao.com/api/search/content/?"+urlencode(data)
    #print(url)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print("请求index页面失败")
        return None


def parse_page_index(html):
    """头条街拍图片存在一个json中,json中的data就是单个图集的信息"""
    data = json.loads(html)
    if data and "data" in data.keys():
        #print("exit")
        for item in data.get("data"):
            #判断是否含有article_url
            if item.get("article_url") and item.get("has_video") == False:
                yield item.get("article_url")
# 自定义获取文本url函数
def get_img_url(text):
    pattern = re.compile(r'http://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')    # 匹配模式
    url_array = re.findall(pattern,text)
    for url in url_array:
        if "pgc-image" in url and "&quot;" in url:
            #print(url)
            #去除url后面不需要的"&quot;"
            url = url.replace("&quot;","")
            #print(url)
            yield url.encode('utf8')


def parse_page_detail(html):
    """解析网站内容，拿到要下载的图片url并下载"""
    res = requests.get(html, headers=headers)
    res.encoding = 'utf-8'
    if res.status_code == 200:
        #print(html)
        for item in  get_img_url(res.text):
            print(item)
            save_image(item)

def save_image(url):
    """解析图片url的信息,把获得的信息写入本地文件中"""
    file_path = '{0}/img/{1}.{2}'.format(os.getcwd(), md5(url).hexdigest(), "jpg")
    print(file_path)
    cont = requests.get(url,stream=True)
    if not os.path.exists(file_path):
        with open(file_path,'wb') as f:
            for bloc in cont.iter_content(1024):
                if bloc:
                    f.write(bloc)


def main(offset):
    """主函数"""
    print("主函数")
    html = get_page_index(20,KEYWORD)
    if html != None:
        for i in parse_page_index(html):
            if re.match(r'http://toutiao.com/group/', i):
                parse_page_detail(i)



if __name__ == "__main__":
    """使用多进程加快速度"""
    #groups = [x * 20 for x in range(GROUP_START, GROUP_END)]
    #print(groups)
    #pool = Pool()
    #pool.map(main, groups)
    print("现在开始龙哥牛逼")
    main(20)











